package org.example

import com.hankcs.hanlp.HanLP
import com.hankcs.hanlp.dictionary.stopword.CoreStopWordDictionary
import com.hankcs.hanlp.tokenizer.StandardTokenizer
import org.apache.spark.sql.SparkSession
import com.hankcs.hanlp.summary.TextRankKeyword

import scala.collection.convert.ImplicitConversions.`collection AsScalaIterable`
import scala.jdk.CollectionConverters.CollectionHasAsScala

object WordCount {
  def main(args: Array[String]): Unit = {
    val spark = SparkSession
      .builder()
      .master("local[*]")
      .appName("sparkBase")
      .getOrCreate()
    val sc = spark.sparkContext
//    英文单词计数
//    sc.textFile("src/main/resources/words.txt")
//      .flatMap(_.split(" ")).map(x => (x, 1)).reduceByKey((x,y) => x +y)
//      .foreach(x => println(x._1 + "," + x._2))
//  中文词语分析
    val chinese = HanLP.segment("严守一没吃早餐，现在特别想吃皮蛋廋肉粥")
//    println(chinese)
//    println(chinese.asScala.map(_.word.trim))
//    标准分词
    val terms = StandardTokenizer.segment("放假++清明节++五一")
//    println(terms)
//    println(terms.asScala.map(_.word.replace("\\s+", "")))
    TextRankKeyword.getKeywordList("速看！广东这5个地方将要拆迁，征收位置、面积、补偿和安置标准都已公布，快来看看有你家吗",5).forEach(println)
    val words = """00:00:00 235102060 [spark大数据] 4 4 https://www.baidu.com"""
      .split("\\s+")
    println(words(2).replaceAll("\\[|\\]",""))

    val textArr = Array(
      "su7爆燃遇难者母亲清空相关微博",
      "甲亢哥被重庆甜妹教育了",
      "胡歌老婆是他前助理",
      "金价再起飞",
      "你能接受与伴侣的年龄差是多少"
    )

    val textRDD = sc.parallelize(textArr)
    val textResult = textRDD.map{
      text =>
        val keyword = TextRankKeyword.getKeywordList(text,5).toString
        val words = transform(text)
        (text ,keyword, words)
    } // RDD[(String, String, List[String])]
    textResult.take(1).foreach(println)
    //Array((男子住进凶宅后家人相继去世,死因离奇,[家人, 相继, 去世, 凶宅, 死因],List(男子, 住进, 凶宅, 家人, 相继, 去世, 死因, 离奇)))
  }

// 结果转换，可以不显示词性
def transform(sentense:String):List[String] ={
  val list = StandardTokenizer.segment(sentense)
  CoreStopWordDictionary.apply(list)
  list.map(x => x.word.replaceAll(" ","")).toList
}
}

